'''
* This is the projet for Brtc LlmOps Platform
* @Author Leon-liao <liaosiliang@alltman.com>
* @Description //TODO 
* @File: 21_study_mmr_search.py
* @Time: 2025/10/30
* @All Rights Reserve By Brtc
'''
import dotenv
import weaviate
from langchain_community.document_loaders import UnstructuredMarkdownLoader
from langchain_openai import OpenAIEmbeddings
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_weaviate import WeaviateVectorStore

dotenv.load_dotenv()
#1、构建加载器一与分割器
loader = UnstructuredMarkdownLoader("./01.项目API文档.md")
text_splitter=RecursiveCharacterTextSplitter(
    separators=[
        "\n\n",
        "\n",
        "。|！|？",
        "\.\s|\!\s|\?\s",  # 英文标点符号后面通常需要加空格
        "；|;\s",
        "，|,\s",
        " ",
        ""
    ],
    is_separator_regex=True,
    chunk_size=500,
    chunk_overlap=50,
    add_start_index=True
)
#2、加载 文档并分割
documents = loader.load()
chunks = text_splitter.split_documents(documents)
client = weaviate.connect_to_local("192.168.106.129", 8080)
#3、将数据存储到向量数据库中
db = WeaviateVectorStore(
    client,
    index_name="TestDemo",
    text_key="text",
    embedding=OpenAIEmbeddings(model="text-embedding-3-small")
)

#ids = db.add_documents(chunks)

#4、转换成检索器
#retriever = db.as_retriever(
#    search_type="similarity_score_threshold",
#    search_kwargs={"k":10, "score_threshold":0.5},
#)

#5、执行检索
docs = db.max_marginal_relevance_search("关于应用配置的接口有那些？")
for one in docs:
    print("=====================================================")
    print(one.page_content[:50])
print(len(docs))
client.close()